package crawler;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.Date;
import java.util.HashSet;

import com.sleepycat.je.rep.monitor.NewMasterEvent;

/**
 * Prepare API list and URL list for crawling
 * DataPreparation
 * @author Lijie Wang, wanglj07@sei.pku.edu.cn
 * 2011-4-7
 */
public class DataPreparation {

	/**
	 * @param args
	 * void
	 */
	public static void main(String[] args) {
		DataPreparation dp = new DataPreparation();
		// TODO Auto-generated method stub
		BufferedReader br;
		try {
			br = new BufferedReader(new FileReader(new File(
					"src/link.list")));
			String line = null;
			String api_name = null;
			HashSet<String> urlList = new HashSet<String>();
			HashSet<String> API_URL_List = new HashSet<String>();
			int numOfAPI = 0;
			int numOfURL = 0;
			
			while ((line = br.readLine()) != null) {
				//read from file
				//this is an api name
				if(line.startsWith("#")){
					//clear the url list of the last api
					API_URL_List.clear();
					api_name = line.trim().substring(1);
					numOfAPI ++;
					//store api
					dp.storeAPI(api_name, numOfAPI);
					continue;
				}
				if(api_name!=null){
					if(urlList.add(line.trim())){
						//store the url
						dp.storeURLContent(line.trim(), null);
					}
					//duplicate elimination on urls of an API
					if(API_URL_List.add(line.trim())){
						dp.storeAPI2URL(numOfAPI, line.trim());
					}
				}
			}
			System.out.println(new Date().toGMTString());
			System.out.println(numOfAPI);
			System.out.println(urlList.size());
			
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
	}
	
	public void storeAPI(String api_name, int api_key){
		System.out.println(api_name+"---"+api_key);
	}
	public void storeAPI2URL(int api_key, String backLinkURL){
//		System.out.println(api_key+"+++"+backLinkURL);
	}
	public void storeURLContent(String url, String content){
//		System.out.println(url+"###"+content);
	}
}
